#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@File    :   website_classification.py
@Contact :   291622538@qq.com

@Modify Time      @Author    @Version    @Description
------------      -------    --------    -----------
2020/11/27 8:48   fan        1.0         None
"""

from scrapy import Spider
from scrapy import Request

from website.items import ClassUrlItem


class ClassUrlGet(Spider):
    # 定义爬虫名称
    name = "class_url_get"

    custom_settings = {
        'ITEM_PIPELINES': {'website.pipelines.SaveClassUrlPipeline': 300},
    }

    def start_requests(self):
        url = "https://www.dmozdir.org/"
        yield Request(url, callback=self.original_parse, dont_filter=False, )

    def original_parse(self, response, **kwargs):
        big_class_list_href = response.xpath('//*[@id="bigClassList"]/ul//a/@href').extract()
        big_class_list = response.xpath('//*[@id="bigClassList"]/ul//a/text()').extract()

        for big_class_href, big_class in zip(big_class_list_href, big_class_list):
            yield Request(url=big_class_href, callback=self.big_class_parse,
                          meta={"big_class": big_class.replace("与", "").replace("/", "、")}, )

    def big_class_parse(self, response):
        second_class_list = response.xpath('//*[@id="mainInner"]/div/ul[2]//a/text()').extract()
        for second_class in second_class_list:
            class_url_item = ClassUrlItem()
            class_url_item["class_url"] = response.meta["big_class"] + "/" + second_class.replace("与", "").replace("/",
                                                                                                                   "、")
            yield class_url_item
